In [ ]:
# Set the device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Load and preprocess the dataset
texts = new_df['content_text'].tolist()
labels = new_df['target'].tolist()
# Count the occurrences of each class
class_counts = Counter(labels)
# Set the minimum number of samples per class
min_samples_per_class = 2
# Filter the classes that have enough samples
valid_classes = [cls for cls, count in class_counts.items() if count >= min_samples_per_class]
# Filter the data and labels based on the valid classes
valid_indices = [i for i, label in enumerate(labels) if label in valid_classes]
valid_texts = [texts[i] for i in valid_indices]
valid_labels = [labels[i] for i in valid_indices]
# Split the data into training and testing sets
train_texts, val_texts, train_labels, val_labels = train_test_split(
valid_texts, valid_labels, test_size=0.2, stratify=valid_labels, random_state=42
)
# Load the BERT tokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
# Tokenize the texts and convert them to input IDs and attention masks
train_encodings = tokenizer(train_texts, truncation=True, padding=True)
val_encodings = tokenizer(val_texts, truncation=True, padding=True)
# Create PyTorch datasets
class CustomDataset(Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
label = self.labels[idx]
label_map = {"TRUE": 0, "FALSE": 1, "UNKNOWN": 2, "MISLEADING": 3, "UNVERIFIED": 4} # Mapping of label values
item["target"] = torch.tensor(label_map[label], dtype=torch.long)
return item
train_dataset = CustomDataset(train_encodings, train_labels)
val_dataset = CustomDataset(val_encodings, val_labels)
# Define the model architecture
model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=5)
model.to(device)
# Define the optimizer and learning rate scheduler
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)
# Define the loss function
loss_fn = nn.CrossEntropyLoss()
# Define the training loop
def train(model, dataloader, loss_fn, optimizer, device):
model.train()
total_loss = 0.0
for batch in dataloader:
input_ids = batch["input_ids"].to(device)
attention_mask = batch["attention_mask"].to(device)
labels = batch["target"].to(device)
optimizer.zero_grad()
outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
loss = outputs.loss
logits = outputs.logits
total_loss += loss.item()
loss.backward()
optimizer.step()
return total_loss / len(dataloader)
# Define the evaluation loop
def evaluate(model, dataloader, loss_fn, device):
model.eval()
total_loss = 0.0
all_predictions = []
all_labels = []
with torch.no_grad():
for batch in dataloader:
input_ids = batch["input_ids"].to(device)
attention_mask = batch["attention_mask"].to(device)
labels = batch["target"].to(device)
outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
loss = outputs.loss
logits = outputs.logits
total_loss += loss.item()
predictions = logits.argmax(dim=1)
all_predictions.extend(predictions.cpu().tolist())
all_labels.extend(labels.cpu().tolist())
return total_loss / len(dataloader), all_predictions, all_labels
# Create data loaders
train_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True)
val_dataloader = DataLoader(val_dataset, batch_size=32)
# Training and evaluation
best_val_loss = float("inf")
best_model_state = None
train_losses = []
val_losses = []
val_predictions = []
val_labels = []
for epoch in range(10): # Modify the number of epochs as needed
train_loss = train(model, train_dataloader, loss_fn, optimizer, device)
val_loss, predictions, labels = evaluate(model, val_dataloader, loss_fn, device)
train_losses.append(train_loss)
val_losses.append(val_loss)
val_predictions.extend(predictions)
val_labels.extend(labels)
val_f1_macro = f1_score(val_labels, val_predictions, average="macro")
val_f1_micro = f1_score(val_labels, val_predictions, average="micro")
val_precision = precision_score(val_labels, val_predictions, average="macro")
val_recall = recall_score(val_labels, val_predictions, average="macro")
print(f"Epoch {epoch+1}:")
print(f" Train Loss: {train_loss:.4f}")
print(f" Validation Loss: {val_loss:.4f}")
print(f" Validation F1 (Macro): {val_f1_macro:.4f}")
print(f" Validation F1 (Micro): {val_f1_micro:.4f}")
print(f" Validation Precision: {val_precision:.4f}")
print(f" Validation Recall: {val_recall:.4f}")
if val_loss < best_val_loss:
best_val_loss = val_loss
best_model_state = model.state_dict()
# Load the best model state
model.load_state_dict(best_model_state)
# Visualization
plt.figure(figsize=(12, 6))
# Loss curve
plt.subplot(1, 2, 1)
plt.plot(train_losses, label="Train")
plt.plot(val_losses, label="Validation")
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.title("Training and Validation Loss")
plt.legend()
# Confusion matrix
plt.subplot(1, 2, 2)
cm = confusion_matrix(val_labels, val_predictions)
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", xticklabels=valid_classes, yticklabels=valid_classes)
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Confusion Matrix")
plt.tight_layout()
plt.show()
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight'] - This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model). - This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model). Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias'] You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. /usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 1: Train Loss: 0.6523 Validation Loss: 0.5036 Validation F1 (Macro): 0.2293 Validation F1 (Micro): 0.8471 Validation Precision: 0.2118 Validation Recall: 0.2500
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 2: Train Loss: 0.4812 Validation Loss: 0.4653 Validation F1 (Macro): 0.3136 Validation F1 (Micro): 0.8524 Validation Precision: 0.3502 Validation Recall: 0.3084
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 3: Train Loss: 0.4384 Validation Loss: 0.4250 Validation F1 (Macro): 0.3442 Validation F1 (Micro): 0.8524 Validation Precision: 0.3371 Validation Recall: 0.3535
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 4: Train Loss: 0.3820 Validation Loss: 0.4070 Validation F1 (Macro): 0.3329 Validation F1 (Micro): 0.8515 Validation Precision: 0.5862 Validation Recall: 0.3333
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 5: Train Loss: 0.3039 Validation Loss: 0.3871 Validation F1 (Macro): 0.3540 Validation F1 (Micro): 0.8534 Validation Precision: 0.4866 Validation Recall: 0.3497
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 6: Train Loss: 0.2301 Validation Loss: 0.4625 Validation F1 (Macro): 0.3673 Validation F1 (Micro): 0.8553 Validation Precision: 0.4841 Validation Recall: 0.3599
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 7: Train Loss: 0.1671 Validation Loss: 0.4096 Validation F1 (Macro): 0.3907 Validation F1 (Micro): 0.8576 Validation Precision: 0.4900 Validation Recall: 0.3818
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 8: Train Loss: 0.1377 Validation Loss: 0.4276 Validation F1 (Macro): 0.4066 Validation F1 (Micro): 0.8594 Validation Precision: 0.4893 Validation Recall: 0.3970
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
Epoch 9: Train Loss: 0.0919 Validation Loss: 0.4647 Validation F1 (Macro): 0.4164 Validation F1 (Micro): 0.8604 Validation Precision: 0.4898 Validation Recall: 0.4078 Epoch 10: Train Loss: 0.0703 Validation Loss: 0.4895 Validation F1 (Macro): 0.4415 Validation F1 (Micro): 0.8624 Validation Precision: 0.6109 Validation Recall: 0.4222
